package com.easy.stock.spider.html;

import java.io.IOException;
import java.nio.charset.Charset;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.jboss.netty.handler.codec.http.HttpResponse;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import com.easy.stock.dao.StockArticleDao;
import com.easy.stock.dao.StockInfoDao;
import com.easy.stock.domain.StockArticle;
import com.easy.stock.domain.StockInfo;
import com.easy.stock.spider.HttpResponseHandler;
import com.easy.stock.spider.SpiderTask;
import com.easy.stock.spider.SpiderTask.STATUS;
import com.easy.stock.util.Constants;
import com.easy.stock.util.JsoupUtil;

public class TheStreetSpider extends BaseHtmlSpider {

	@Autowired
	private StockInfoDao stockInfoDao;
	
	@Autowired
	private StockArticleDao stockArticleDao;
	
	private static String SITE_URL="http://www.thestreet.com";
	private String baseUrl;
	
	public void setBaseUrl(String baseUrl) {
		this.baseUrl = baseUrl;
	}

	
	@Override
	protected List<SpiderTask> prepareSeeds() {
		List<SpiderTask> tasks = new ArrayList<SpiderTask>();
		List<StockInfo> stocks = null;
		int limit=1000;
		int offset=0;
		while(!(stocks=stockInfoDao.getStockInfoList(offset, limit)).isEmpty()) {
			for(StockInfo stock : stocks) {
				SpiderTask task = new SpiderTask(baseUrl+stock.getId()+".html?omorig=header", new DetailHandler(this, stock.getId()), STATUS.READY);
				tasks.add(task);
			}
			
			offset+=limit;
		}
		return tasks;
	}
	
	public static class DetailHandler extends HttpResponseHandler {

		private String stockId;
		
		public DetailHandler(BaseHtmlSpider spider, String stockId) {
			super.spider = spider;
			this.stockId = stockId;
		}
		
		@Override
		public List<SpiderTask> doOnCompleted(HttpResponse response)
				throws IOException {
			
			TheStreetSpider mySpider = (TheStreetSpider)spider;
			String content = response.getContent().toString(Charset.forName("UTF-8"));
			Document doc = Jsoup.parse(content);
			
			List<SpiderTask> tasks = new ArrayList<SpiderTask>();
			Elements elems = doc.select("div.linkBoxes > ul > li");
			int elemNum = elems.size();
			for(int i=0; i<elemNum; i++) {
				Element elem = elems.get(i);
				Element titleElem = elem.select("span a").get(0);
				String url = titleElem.attr("href");
				
				// 其他网站的报道，先跳过
				if(url.indexOf("http://")>=0)
					continue;
				
				String articleUrl = SITE_URL+url;
				if(!mySpider.stockArticleDao.isArticleExists(articleUrl)) {
					SpiderTask task = new SpiderTask(articleUrl, new StreetArticleHandler(spider, stockId, articleUrl), STATUS.READY);
					tasks.add(task);
				}
			}
			
			LOG.info("finish crawl stock article page, stockId="+stockId+", articles="+tasks.size());
			return tasks;
		}
		
		@Override
		public void doOnThrowable(Throwable t) {
			// do nothing
		}
		
	}
	
	public static class StreetArticleHandler extends HttpResponseHandler {

		private String stockId;
		private String url;
		
		public StreetArticleHandler(BaseHtmlSpider spider, String stockId, String url) {
			super.spider = spider;
			this.stockId = stockId;
			this.url = url;
		}
		
		@Override
		public List<SpiderTask> doOnCompleted(HttpResponse response)
				throws IOException {
			
			TheStreetSpider mySpider = (TheStreetSpider)spider;
			String content = response.getContent().toString(Charset.forName("UTF-8"));
			Document doc = Jsoup.parse(content);
			
			StockArticle article = new StockArticle();
			article.setUrl(url);
			String author = JsoupUtil.getElemText(doc, "#storyAuthorLink a", false);
			if(author.length()==0) {
				// 缺失作者不抓取
				return null;
			}
			article.setAuthor(author);
			article.setAuthor_company("");
			article.setAuthor_desc("");
			article.setTitle(JsoupUtil.getElemHtml(doc, "#storyHdr h1", true));
			
			SimpleDateFormat sdf = new SimpleDateFormat("MM/dd/yy - HH:mm aa", Locale.US);
			String dateStr = JsoupUtil.getElemText(doc, "#storyAuthorLink span:last-of-type", true);
			dateStr = dateStr.substring(0, dateStr.lastIndexOf(" "));
			try {
				article.setDate(sdf.parse(dateStr));
			} catch (ParseException e) {
				throw new RuntimeException("article prase date error, str="+dateStr);
			}
			

			article.setSummary("");
			
			Element contentElem = doc.select("div.virtualpage").first();
			// 删除其他干扰的元素
			contentElem.select("div").remove();
			article.setContent(contentElem.html());
			
			article.setSrc(Constants.THE_STREET);
			article.setStock_id(stockId);
			
			mySpider.stockArticleDao.addArticle(article);
			LOG.info("crawler article success, stockId="+stockId+", url="+url);
			return null;
		}
		
		
		@Override
		public void doOnThrowable(Throwable t) {
			// do nothing
		}
	}
}
